package com.shujia.sql

import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}

object Demo10SparkSqlOnSubmit {
  def main(args: Array[String]): Unit = {

    val spark: SparkSession = SparkSession
      .builder()
      .appName("sql")
      .config("spark.sql.shuffle.partitions", 1)
      .getOrCreate()


    val studentDF: DataFrame = spark.read
      .option("sep", ",")
      .schema("id STRING,name STRING,age INT,gender STRING,clazz STRING")
      .csv("/data/students") //读取hdfs上的数据


    studentDF.createOrReplaceTempView("student")


    val resultDF: DataFrame = spark.sql(
      """
        |
        |select clazz,count(1) from student group by clazz
        |
        |
      """.stripMargin)


    resultDF
      .write
      .option("sep", "\t")
      .mode(SaveMode.Overwrite)
      .csv("/data/student/clazz_num")


  }
}
